package zsd.reiview.experiment.crfTest;

import java.io.File;
import java.io.IOException;
import java.util.Random;

import org.apache.log4j.Logger;

import zsd.reiview.experiment.crf.CrfBioTagChunkCodec;
import zsd.reiview.experiment.crf.CrfChunkerTest;
import zsd.reiview.experiment.crf.CrfCorpus;
import zsd.reiview.experiment.crf.CrfFeatureExtractor;
import zsd.reiview.experiment.crf.GraphRank;
import zsd.reiview.experiment.crf.IctSplitTokenizerFactory;
import zsd.reiview.experiment.crf.CrfChunkerTest.Result;
import zsd.reiview.experiment.crf.CrfCorpus.COAE2009TargetCorpus;
import zsd.reiview.experiment.crf.CrfCorpus.Task3Corpus;
import zsd.reiview.utility.FileUtility;

import com.aliasi.chunk.ChunkerEvaluator;
import com.aliasi.chunk.Chunking;
import com.aliasi.chunk.ChunkingEvaluation;
import com.aliasi.chunk.TagChunkCodec;
import com.aliasi.corpus.ListCorpus;
import com.aliasi.corpus.XValidatingObjectCorpus;
import com.aliasi.crf.ChainCrfChunker;
import com.aliasi.crf.ChainCrfFeatureExtractor;
import com.aliasi.io.LogLevel;
import com.aliasi.io.Reporter;
import com.aliasi.io.Reporters;
import com.aliasi.stats.AnnealingSchedule;
import com.aliasi.stats.RegressionPrior;
import com.aliasi.tokenizer.CharacterTokenizerFactory;
import com.aliasi.tokenizer.TokenNGramTokenizerFactory;
import com.aliasi.tokenizer.TokenizerFactory;
import com.aliasi.util.AbstractExternalizable;

/**
 * @Title: TranTestListChunkerTest.java
 * @Package zsd.reiview.experiment.crfTest
 * @Description: TODO(添加描述)
 * @author 朱圣代
 * @date 2011-11-8 下午06:50:48
 * @version V1.0
 */
public class TranTestListChunkerTest {

	/**
	 * @Title: main
	 * @Description:
	 * @param @param args
	 * @return void
	 * @throws
	 * @date 2011-11-8 下午06:50:48
	 * 
	 * 
	 */
	boolean updateCrf = true;
	ListCorpus<Chunking> trainCorpus;
	ListCorpus<Chunking> testCorpus;
	TokenizerFactory trainTokenizerFactory = new IctSplitTokenizerFactory();
	ChainCrfChunker crf;

	private static final Logger log = Logger.getLogger(TranTestListChunkerTest.class);

	String name;

	public String[] getPath(String dir, String name, int N) throws IOException {
		String[] paths = new String[N];
		for (int i = 0; i < N; i++) {
			paths[i] = dir + "\\" + name + "_" + i + ".txt";
			FileUtility.createFile(paths[i]);
		}
		return paths;
	}

	public ChainCrfChunker trainCrf() throws Exception {
		ChainCrfFeatureExtractor<String> featureExtractor = new CrfFeatureExtractor();

		boolean addIntercept = true;

		int minFeatureCount = 1;

		boolean cacheFeatures = true;

		boolean allowUnseenTransitions = true;

		double priorVariance = 4.0;
		boolean uninformativeIntercept = true;
		RegressionPrior prior = RegressionPrior.gaussian(priorVariance, uninformativeIntercept);
		int priorBlockSize = 3;

		double initialLearningRate = 0.05;
		double learningRateDecay = 0.995;
		AnnealingSchedule annealingSchedule = AnnealingSchedule.exponential(initialLearningRate, learningRateDecay);

		double minImprovement = 0.00001;
		int minEpochs = 2;
		int maxEpochs = 2000;

		Reporter reporter = Reporters.stdOut().setLevel(LogLevel.DEBUG);

		System.out.println("\nEstimating");

		TagChunkCodec codec = new CrfBioTagChunkCodec(trainTokenizerFactory, true);

		File modelFile = new File(getPath("myDataBase\\models\\" + name, name, 1)[0]);

		if (updateCrf) {
			crf = ChainCrfChunker.estimate(trainCorpus, codec, trainTokenizerFactory, featureExtractor, addIntercept, minFeatureCount, cacheFeatures, prior, priorBlockSize, annealingSchedule,
					minImprovement, minEpochs, maxEpochs, reporter);

			System.out.println("\nCompiling to file=" + modelFile);
			AbstractExternalizable.serializeTo(crf, modelFile);
		} else {
			crf = (ChainCrfChunker) AbstractExternalizable.readObject(modelFile);
		}
		return crf;
	}

	public Result testAll2011() throws Exception {
		Task3Corpus task3Corpus = new Task3Corpus();
		Result result2 = new Result(name);
		testCorpus = task3Corpus.getTestTask3Corpus(1, 10000, "D", true, false);

		ChunkerEvaluator chunkerEvaluator = new ChunkerEvaluator(crf);

		testCorpus.visitTest(chunkerEvaluator);
		ChunkingEvaluation chunkingEvaluation = chunkerEvaluator.evaluation();
		result2.add(chunkingEvaluation.precisionRecallEvaluation());

		testCorpus = task3Corpus.getTestTask3Corpus(1, 10000, "E", true, false);
		testCorpus.visitTest(chunkerEvaluator);
		ChunkingEvaluation chunkingEvaluation2 = chunkerEvaluator.evaluation();
		result2.add(chunkingEvaluation2.precisionRecallEvaluation());

		testCorpus = task3Corpus.getTestTask3Corpus(1, 10000, "F", true, false);
		testCorpus.visitTest(chunkerEvaluator);
		ChunkingEvaluation chunkingEvaluation3 = chunkerEvaluator.evaluation();
		result2.add(chunkingEvaluation3.precisionRecallEvaluation());

		log.error(result2.toMatricString());

		return result2;
	}

	double alph, beita, gama, alph2;
	int graph;
	int trainSplit;
	int trainMaxNum;

	public void test09_11GraphedRanked() throws Exception {
		COAE2009TargetCorpus COAE2009TargetCorpus = new COAE2009TargetCorpus();
		ListCorpus<Chunking> listTran = COAE2009TargetCorpus.get2009TrainTargetCorpus(trainSplit, 10000);
		Task3Corpus testTask3Corpus = new Task3Corpus();
		ListCorpus<Chunking> test;
		test = testTask3Corpus.getTestTask3Corpus(1, 10000, "D", true, false);
		GraphRank graphRank = new GraphRank();
		graphRank.alph2 = alph2;
		boolean updateFilter = false;
		int type = -1;
		trainCorpus = graphRank.getListRankedCorpus(graph, "test_0_1_2", type, trainMaxNum, alph, beita, gama, listTran, test, 3, updateFilter, trainSplit);
		name = graph + "_Graph_" + trainMaxNum + "_" + trainSplit + "_graph_filter_" + alph + "_" + alph2 + "_" + beita + "_" + gama;
		trainCrf();
		testAll2011();
	}

	public void testAll09_11Ranked() throws Exception {
		trainSplit = 2;
		trainMaxNum = 5000;
		alph = 0.3;
		beita = 0.9;
		gama = 0.5;

		graph = 0;
		test09_11GraphedRanked();
		graph = 1;
		test09_11GraphedRanked();
		graph = 2;
		test09_11GraphedRanked();
	}

	public static void main(String[] args) throws Exception {

		TranTestListChunkerTest test = new TranTestListChunkerTest();
		test.testAll09_11Ranked();
	}

}
